sarc:
  health_monitor:
    directory: check-results

    parameterizations:
      cluster_name:
        - mila
        - narval

    checks:
      at_least_one_{cluster_name}:
        class: sarc.alerts.checks:FilterCheck
        active: true
        interval: 1h
        min_count: 1
        filters:
          - cluster_name == params.cluster_name
          - ran_for(hours=5)
          - allocated_gpu()

      has_gpu_util_{cluster_name}:
        class: sarc.alerts.checks:FilterCheck
        active: false
        interval: 1s
        min_count: 1
        filters:
          - cluster_name == params.cluster_name
          - ran_for(hours=5)
          - stored_statistics?.gpu_utilization?.mean > 0
        depends: "at_least_one_{cluster_name}"

# Use this to test at a specific time and make it so that every sleep will take 0.5 seconds
time:
  class: FrozenTime
  time: "2023-10-08T15:47:47Z"
  sleep_beat: 0.5